package com.foreveross.crawl.adapter.sub.impl20140402.v3;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BestMatchSpecFactory;
import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.foreveross.crawl.adapter.AbstractAdapter;
import com.foreveross.crawl.adapter.PlaneInfoEntityBuilder;
import com.foreveross.crawl.common.exception.self.PageErrorResultException;
import com.foreveross.crawl.common.exception.self.UnableParseRouteTypeException;
import com.foreveross.crawl.common.util.RandomBrowserVersion;
import com.foreveross.crawl.common.util.RegHtmlUtil;
import com.foreveross.crawl.domain.airfreight.AbstractPlaneInfoEntity;
import com.foreveross.crawl.domain.airfreight.CabinEntity;
import com.foreveross.crawl.domain.airfreight.TransitEntity;
import com.foreveross.crawl.domain.airfreight.doub.CabinRelationEntity;
import com.foreveross.crawl.domain.airfreight.doub.DoublePlaneInfoEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnCabinEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnDoublePlaneInfoEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnTransitEntity;
import com.foreveross.crawl.domain.airfreight.single.SinglePlaneInfoEntity;
import com.foreveross.crawl.exception.FlightInfoNotFoundException;
import com.foreveross.crawl.util.DateUtil;
import com.foreveross.taskservice.common.bean.TaskModel;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;

/**
 * 国泰航空适配器。
 * 
 * @author luomingliang@foreveross.com
 * @version 1.0.0
 * @date 2014/08/05
 */
public class CathayPacificAdapter extends AbstractAdapter {
	
	private HttpClientContext context=null;
	
	/**
	 * 查询航班详细信息失败尝试次数
	 */
	private static final int queryflightTimes = 1;
	
	/**
	 * 每个舱位等级失败尝试次数
	 */
	private static final int queryCabinTimes=2;
	
	/**
	 * 往返航班抓取前 5条 和最后一条 
	 */
	private static final int grabsize=9;
	
	/**
	 * 返程抓取条数
	 */
	private static final int grabreturnsize=4;
	
	/**
	 * 舱位 现在只拿经济舱
	 */
	private final static Map<String, String> CABIN = ImmutableMap.of("Y", "经济");
	//private final static Map<String, String> CABIN = ImmutableMap.of("Y", "经济", "W", "特选经济", "C", "公务","F", "头等" );
	
	
	
	public CathayPacificAdapter(TaskModel taskQueue) {
		super(taskQueue);
	}

	@Override
	public String getUrl() throws Exception {
		return null;
	}

	@Override
	public Object fetch(String url) throws Exception {
		switch (super.getRouteType()) {
		case DOMESTIC_ONEWAYTRIP:
		case INTERNATIONAL_ONEWAY:
			return fetchInterOneWay();
		case DOMESTIC_ROUNDTRIP:
		case INTERNATIONAL_ROUND:
			return fetchInterround();
		default:
			throw new UnableParseRouteTypeException(taskQueue, getRouteType().getName());
		}
	}



	@Override
	public List<Object> paraseToVo(Object fetchObject) throws Exception {
		if(fetchObject==null){
			throw new Exception("数据抓取失败");
		}
		List list = (List)fetchObject;
		if(list.size()==0){
			throw new Exception("数据抓取失败");
		}
		return list;
	}
	
	/**
	 * 单程
	 * @return
	 */
	private Object fetchInterOneWay() {
		List<SinglePlaneInfoEntity> result=new ArrayList<SinglePlaneInfoEntity>();
		for (String cabinType : CABIN.keySet()) {
			List<SinglePlaneInfoEntity> grabList=fetchSinglePageDates(cabinType);
			if(null==result || result.size()==0)
				result=grabList;
			else 
				this.hanbinListSingle(result,grabList);
		}
		return result;
	}
	
	
	/**
	 * 单程
	 * @param cabinType
	 * @return
	 */
	private List<SinglePlaneInfoEntity> fetchSinglePageDates(String cabinType) {
		List<SinglePlaneInfoEntity> listSigle=new ArrayList<SinglePlaneInfoEntity>();
		for(int j=0;j<queryCabinTimes;j++){
			listSigle.clear();
			try {
				String page=null;
				for(int i=0;i<queryflightTimes;i++){
					try {
						page = fetchStep1(cabinType);
						break;
					} catch (Exception e2) {
						e2.printStackTrace();
						if(i==queryflightTimes-1)
							throw e2;
					}
				}
				Document doc = Jsoup.parse(page);
				Elements froms=doc.select("form[name=AVAI_FORM]");
				Element goElement = doc.getElementById("mainTabContent0"); // 去程航班
				Elements  gorows=goElement.select("input[name=ROW_1]");
				for (Element e : gorows) {
					for(int i=0;i<queryflightTimes;i++){
						try {
							String row1=e.attr("value");
							String pages=fetchDetail(froms.get(0),row1,null);
							Document doctail = Jsoup.parse(pages);
							Element body=doctail.getElementById("business");
							SinglePlaneInfoEntity sigle = this.changeToSingle(body, cabinType);
							listSigle.add(sigle);
							break;
						} catch (Exception e1) {
							logger.info(e1);
							logger.info("请求往返数据航班第：" + (i + 1) + "次失败");
						}
					}	
				}
				break;
			} catch (Exception e) {
				logger.info(e);
				logger.info("请求往舱位"+CABIN.get(cabinType)+"第：" + (j + 1) + "次失败");
			}
		}
		return listSigle;
	}
	
	/**
	 * 将网页数据转换为单程对象
	 * @param body
	 * @param cabinType
	 * @return
	 * @throws Exception
	 */
	private SinglePlaneInfoEntity changeToSingle(Element body, String cabinType) throws Exception {
		Elements flightElement =body.getElementsByTag("table");
		Element table=flightElement.get(0);
		List<TransitEntity> listTraint= htmlToTraintEntity(table.getElementsByClass("odd1"),TransitEntity.class);
		if(listTraint.isEmpty())
			return null;
		Element priceTable=flightElement.get(1);
		Elements tds=priceTable.getElementsByClass("odd1").get(0).getElementsByTag("td");
		String[] priceStr=tds.get(1).text().split(" ");
		String currency=priceStr[0];
		Double lowerPrice=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		
		priceStr=tds.get(2).text().split(" ");
		Double taxesPrice=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		priceStr=tds.get(3).text().split(" ");
		taxesPrice+=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		
		priceStr=tds.get(4).text().split(" ");
		Double sumPrice=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		
		TransitEntity trainsit=listTraint.get(0);
		
		SinglePlaneInfoEntity entity = this.buildPlaneInfo(taskQueue,
				trainsit.getCarrierKey(), trainsit.getCarrierName(),
				trainsit.getCarrierFullName(), trainsit.getStartTime(),
				trainsit.getEndTime(), trainsit.getFlightNo(), lowerPrice,
				lowerPrice, trainsit.getFlightType(), sumPrice, sumPrice,
				taxesPrice, taxesPrice, SinglePlaneInfoEntity.class);
		entity.setCurrency(currency);
		
		//去程中转
		if(listTraint.size()>1){
			listTraint.get(0).setStayTime(null);
			entity.setTransits((ArrayList<TransitEntity>) listTraint);
		}
		
		//去程舱位
		CabinEntity cabin = PlaneInfoEntityBuilder.buildCabinInfo(
				CABIN.get(cabinType), cabinType, CABIN.get(cabinType), taxesPrice.toString(),
				lowerPrice.toString(), sumPrice.toString(), null, null, CabinEntity.class);
		Set<CabinEntity> cabins = new HashSet<CabinEntity>();
		cabins.add(cabin);
		entity.setCabins(cabins);
		return entity;
	}

	/**
	 * 国际往返
	 */
	private Object fetchInterround() throws Exception {
		
		List<DoublePlaneInfoEntity> result = new ArrayList<DoublePlaneInfoEntity>();
		for (String cabinType : CABIN.keySet()) {
			List<DoublePlaneInfoEntity> grabList=fetchPageDatas(cabinType);
			if(null==result || result.size()==0)
				result=grabList;
			else 
				this.hanbinListDouble(result,grabList);
		}
		
		try{
			PlaneInfoEntityBuilder.buildLimitPrice(result);
		}catch(Exception e){
			logger.info("国际往返排序错误，请检查数据");
			logger.info(e);
			try{
				logger.info(result);
			}catch(Exception e1){
				logger.info("打印数据出错");
			}
		}
		if(result.size()==0)
			throw new Exception("抓取错误!");
		return result;
	}
	
	
	
	
	/**
	 * 获得返程某舱位的航班集合,如果第一条数据抓取失败则默认为全部抓取失败
	 * @param cabinType
	 * @return
	 * @throws Exception
	 */
	private List<DoublePlaneInfoEntity> fetchPageDatas(String cabinType) throws Exception {

		List<DoublePlaneInfoEntity> listDouble=new ArrayList<DoublePlaneInfoEntity>();
		int fromAll=0;
		int toAll=0;
		
	
		String page=null;
		for(int i=0;i<queryflightTimes;i++){
			try {
				//抓取第一步第二步
				page = fetchStep1(cabinType);
				break;
			} catch (Exception e2) {
				if(!isUseProxyip() || proxyIp.getIp().startsWith("112.124"))
					throw e2;
				if(i==queryflightTimes-1)
					throw e2;
				Thread.sleep(isUseProxyip()?1000*10:60*1000);
				e2.printStackTrace();
			}
		}
		Document doc = Jsoup.parse(page);
		for (int j = 0; j < queryflightTimes; j++) {
			listDouble.clear();
			try {
				Elements froms = doc.select("form[name=AVAI_FORM]");
				Element goElement = doc.getElementById("mainTabContent0"); // 去程航班
				Element reElement = doc.getElementById("mainTabContent1"); // 回程航班
				Elements gorows = goElement.select("input[name=ROW_1]");
				Elements torows = reElement.select("input[name=ROW_2]");
				int fromcount=1;
				fromAll=gorows.size();
				toAll=torows.size();
				for (Element e : gorows) {
					if(fromcount>grabsize && fromcount!=fromAll){
						fromcount++;
						continue;
					}
					try {
						String row1 = e.attr("value");
						DoublePlaneInfoEntity doub = null;
						int tocount=1;
						for (Element re : torows) {
							if(tocount>grabreturnsize && tocount!=toAll){
								tocount++;
								continue;
							}
							//获得详细信息，失败尝试次数为 queryflightTimes
							for (int i = 0; i < queryflightTimes; i++) {
								try {
									String row2 = re.attr("value");
									String pages = fetchDetail(froms.get(0), row1,
											row2);
									Document doctail = Jsoup.parse(pages);
									Element body = doctail
											.getElementById("business");
									if (null == doub)
										doub = this.changeToDoube(body, cabinType);
									else
										this.henbinReturnEn(doub,this.changeToDoube(body, cabinType));
									break;
								} catch (Exception e1) {
									
									logger.error(e1.getStackTrace());
									logger.info("请求往返数据航班第：" + (i + 1) + "次失败");
								}
							}
							if(tocount==1 && null==doub){
								//第一条抓取不成功刚默认为失败
								throw new PageErrorResultException("抓取页面数据失败");
							}
							tocount++;
						}
						if (null != doub) {
							listDouble.add(doub);
						}
					} catch (Exception e1) {
						e1.printStackTrace();
					}
					
					if(listDouble.size()==0 && fromcount==1){
						throw new PageErrorResultException("抓取页面数据失败");//第一条抓取不成功刚默认为失败
					}
					fromcount++;
				}
				break;
			} catch (Exception e) {
				if(!isUseProxyip() || proxyIp.getIp().startsWith("112.124"))
					throw e;
				if(j==queryflightTimes-1)
					throw e;
				
				logger.error(e.getStackTrace());
				logger.info("请求往舱位" + CABIN.get(cabinType) + "第：" + (j + 1)
						+ "次失败");
			}
		}
		if(fromAll>0 && toAll>0 && listDouble.size()==0)
			throw new Exception("抓取页面数据失败");
		
		return listDouble;
	}
	
	/**
	 * 合并返程
	 * @param doub
	 * @param changeToDoube
	 */
	private void henbinReturnEn(DoublePlaneInfoEntity doub,
			DoublePlaneInfoEntity newDoub) {
		if(null==newDoub)
			return;
		doub.getCabinRelations().addAll(newDoub.getCabinRelations());
		doub.getReturnPlaneInfos().addAll(newDoub.getReturnPlaneInfos());
		
	}

	/**
	 * 将网页数据转换为 DoublePlaneInfoEntity 对象
	 * @param body
	 * @return
	 * @throws Exception 
	 */
	private DoublePlaneInfoEntity changeToDoube(Element body,String cabinType) throws Exception {
		
		Elements flightElement =body.getElementsByTag("table");
		Element table=flightElement.get(0);
		List<TransitEntity> listTraint= htmlToTraintEntity(table.getElementsByClass("odd1"),TransitEntity.class);
		if(listTraint.isEmpty())
			return null;
		List<ReturnTransitEntity> listReturnTaint=htmlToTraintEntity(table.getElementsByClass("even1"),ReturnTransitEntity.class);
		if(listReturnTaint.isEmpty())
			return null;
		
		Element priceTable=flightElement.get(1);
		Elements tds=priceTable.getElementsByClass("odd1").get(0).getElementsByTag("td");
		String[] priceStr=tds.get(1).text().split(" ");
		String currency=priceStr[0];
		Double lowerPrice=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		
		priceStr=tds.get(2).text().split(" ");
		Double taxesPrice=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		priceStr=tds.get(3).text().split(" ");
		taxesPrice+=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		
		priceStr=tds.get(4).text().split(" ");
		Double sumPrice=PlaneInfoEntityBuilder.getDouble(priceStr[1]);
		
		TransitEntity trainsit=listTraint.get(0);
		TransitEntity lasttrainsit=listTraint.get(listTraint.size()-1);
		DoublePlaneInfoEntity entity = this.buildPlaneInfo(taskQueue,
				trainsit.getCarrierKey(), trainsit.getCarrierName(),
				trainsit.getCarrierFullName(), trainsit.getStartTime(),
				lasttrainsit.getEndTime(), trainsit.getFlightNo(), lowerPrice,
				lowerPrice, trainsit.getFlightType(), sumPrice, sumPrice,
				taxesPrice, taxesPrice, DoublePlaneInfoEntity.class);
		entity.setCurrency(currency);
		entity.setFlightDuration(trainsit.getStayTime());
		
		//去程中转
		if(listTraint.size()>1){
			listTraint.get(0).setStayTime(null);
			entity.setTransits((ArrayList<TransitEntity>) listTraint);
		}
		
		//去程舱位
		CabinEntity cabin = PlaneInfoEntityBuilder.buildCabinInfo(
				CABIN.get(cabinType), cabinType, CABIN.get(cabinType), null,
				null, null, null, null, CabinEntity.class);
		Set<CabinEntity> cabins = new HashSet<CabinEntity>();
		cabins.add(cabin);
		entity.setCabins(cabins);
		
		//返程
		ReturnTransitEntity returnTrainsit=listReturnTaint.get(0);
		ReturnTransitEntity lastReturnTrainsit=listReturnTaint.get(listReturnTaint.size()-1);
		ReturnDoublePlaneInfoEntity returnEntity=this.buildPlaneInfo(taskQueue,
				returnTrainsit.getCarrierKey(), returnTrainsit.getCarrierName(),
				returnTrainsit.getCarrierFullName(), returnTrainsit.getStartTime(),
				lastReturnTrainsit.getEndTime(), returnTrainsit.getFlightNo(), lowerPrice,
				lowerPrice, returnTrainsit.getFlightType(), sumPrice, sumPrice,
				taxesPrice, taxesPrice, ReturnDoublePlaneInfoEntity.class);
		returnEntity.setFlightDuration(returnTrainsit.getStayTime());
		//返程舱位
		Set<ReturnCabinEntity> returnCabins =new HashSet<ReturnCabinEntity>();
		ReturnCabinEntity returnCabin=PlaneInfoEntityBuilder.buildCabinInfo(
				CABIN.get(cabinType), cabinType, CABIN.get(cabinType), null,
				null, null, null, null, ReturnCabinEntity.class);
		returnCabins.add(returnCabin);
		returnEntity.setReturnCabins(returnCabins);
		
		
		//返程中转
		if(listReturnTaint.size()>1){
			listReturnTaint.get(0).setStayTime(null);
			returnEntity.setReturnTransits((ArrayList<ReturnTransitEntity>) listReturnTaint);
		}
		
		Set<ReturnDoublePlaneInfoEntity> returnPlaneInfos = new HashSet<ReturnDoublePlaneInfoEntity>();
		returnPlaneInfos.add(returnEntity);
		entity.setReturnPlaneInfos(returnPlaneInfos);
		
		
		//舱位对应关系
		Set<CabinRelationEntity> cabinRelations = new HashSet<CabinRelationEntity>();
		CabinRelationEntity relation=new CabinRelationEntity();
		relation.setCabinId(cabin.getId());
		relation.setReturnCabinId(returnCabin.getId());
		relation.setFullPrice(lowerPrice);
		relation.setTaxesPrice(taxesPrice);
		relation.setTotalFullPrice(sumPrice);
		relation.setPlaneInfoEntity(entity);
		cabinRelations.add(relation);
		
		entity.setCabinRelations(cabinRelations);
		
		return entity;
	}
	
	/**
	 * 合并单程不同舱位抓到的list集合
	 * @param result
	 * @param grabList
	 */
	private void hanbinListSingle(List<SinglePlaneInfoEntity> result,
			List<SinglePlaneInfoEntity> grabList) {
		if(null==grabList || grabList.size()==0)
			return;
		List<SinglePlaneInfoEntity> list =new ArrayList<SinglePlaneInfoEntity>();
		for(SinglePlaneInfoEntity orig:grabList){
			boolean isexit=false;
			String origTrip=this.findTrip(orig);
			for(SinglePlaneInfoEntity dest:result){
				String destTrip=this.findTrip(dest);
				if(origTrip.equals(destTrip)){
					isexit=true;
					dest.getCabins().addAll(orig.getCabins());
					break;
				}
			}
			if(!isexit){
				list.add(orig);
			}
		}
		result.addAll(list);
		
	}
	
	/**
	 * 合并往返不同舱位抓到的list集合
	 * @param result
	 * @param grabList
	 */
	private void hanbinListDouble(List<DoublePlaneInfoEntity> result,
			List<DoublePlaneInfoEntity> grabList) {
		
		if(null==grabList || grabList.size()==0)
			return;
		
		List<DoublePlaneInfoEntity> newList=new ArrayList<DoublePlaneInfoEntity>();
		for(DoublePlaneInfoEntity orig:grabList){
			boolean isexit=false;
			String origTrip=this.findTrip(orig);
			for(DoublePlaneInfoEntity dest:result){
				String destTrip=this.findTrip(dest);
				if(origTrip.equals(destTrip)){
					isexit=true;
					this.hanbinDouble(dest,orig);
					break;
				}
			}
			if(!isexit){
				newList.add(orig);
			}
		}
		result.addAll(newList);
	}
	
	/**
	 * 合并舱位不同的doule
	 * @param dest
	 * @param orig
	 */
	private void hanbinDouble(DoublePlaneInfoEntity dest,
			DoublePlaneInfoEntity orig) {
		dest.getCabinRelations().addAll(orig.getCabinRelations());
		dest.getCabins().addAll(orig.getCabins());
		if(null!=dest.getReturnPlaneInfos() && null!=orig.getReturnPlaneInfos()){
			Set<ReturnDoublePlaneInfoEntity> returnPlaneInfos = new HashSet<ReturnDoublePlaneInfoEntity>();
			for(ReturnDoublePlaneInfoEntity origretun:orig.getReturnPlaneInfos()){
				boolean isexit=false;
				String origTrip=this.findTrip(origretun);
				for(ReturnDoublePlaneInfoEntity destretun:dest.getReturnPlaneInfos()){
					String destTrip=this.findTrip(destretun);
					if(origTrip.equals(destTrip)){
						isexit=true;
						destretun.getReturnCabins().addAll(origretun.getReturnCabins());
						break;
					}
				}
				if(!isexit){
					returnPlaneInfos.add(origretun);
				}
			}
			dest.getReturnPlaneInfos().addAll(returnPlaneInfos);
		}
	}

	/**
	 * 获得 AbstractPlaneInfoEntity 的航程  如
	 * @param entity
	 * @return
	 */
	private <T extends AbstractPlaneInfoEntity> String findTrip(T entity) {
		StringBuffer trip=new StringBuffer();
		
		if(entity.getClass().equals(ReturnDoublePlaneInfoEntity.class)){
			ReturnDoublePlaneInfoEntity returnEntity=(ReturnDoublePlaneInfoEntity) entity;
			if(null!=returnEntity.getReturnTransits() &&returnEntity.getReturnTransits().size()>0){
				for(ReturnTransitEntity train: returnEntity.getReturnTransits()){
					trip.append("-"+train.getFlightNo());
				}
			}
		}else if(entity.getClass().equals(DoublePlaneInfoEntity.class)){
			DoublePlaneInfoEntity doubleEntity=(DoublePlaneInfoEntity) entity;
			if(null!=doubleEntity.getTransits() && doubleEntity.getTransits().size()>0){
				for(TransitEntity train:doubleEntity.getTransits()){
					trip.append("-"+train.getFlightNo());
				}
			}
		}else if(entity.getClass().equals(SinglePlaneInfoEntity.class)){
			SinglePlaneInfoEntity single=(SinglePlaneInfoEntity) entity;
			if(null!=single.getTransits() && single.getTransits().size()>0){
				for(TransitEntity train:single.getTransits()){
					trip.append("-"+train.getFlightNo());
				}
			}
		}
		if(trip.length()==0){
			trip.append(entity.getFlightNo());
		}
		return trip.toString();
	}


	
	/**
	 * 创建对象
	 * @param task
	 * @param carrierKey
	 * @param carrierName
	 * @param carrierFullName
	 * @param startTime
	 * @param endTime
	 * @param flightNo
	 * @param lowerPrice
	 * @param highestPrice
	 * @param flightType
	 * @param sumPrice
	 * @param highestSumPrice
	 * @param taxesPrice
	 * @param highestTaxesPrice
	 * @param clzz
	 * @return
	 * @throws Exception
	 */
	private <T extends AbstractPlaneInfoEntity> T buildPlaneInfo(TaskModel task,
			String carrierKey, String carrierName, String carrierFullName,
			Date startTime, Date endTime, String flightNo, Double lowerPrice,
			Double highestPrice, String flightType, Double sumPrice,
			Double highestSumPrice, Double taxesPrice, Double highestTaxesPrice,
			Class<T> clzz) throws Exception {
			AbstractPlaneInfoEntity entity = clzz.newInstance();
			entity.setFlightNo(flightNo);
			//entity.setFlightDate(flightDate);
			entity.setCarrierKey(carrierKey);
			entity.setCarrierName(carrierName);
			entity.setCarrierFullName(carrierFullName);
			entity.setCreateTime(new Date());
			entity.setEndTime(endTime);
			entity.setFlightNo(flightNo);
			entity.setFlightType(flightType);
			entity.setStartTime(startTime);
			entity.setTotalLowestPrice(lowerPrice);
			entity.setTotalHighestPrice(highestPrice);
			entity.setTotalLowestTaxesPrice(taxesPrice);
			entity.setTotalHighestTaxesPrice(highestTaxesPrice);
			entity.setSumLowestPrice(sumPrice);
			entity.setSumHighestPrice(highestSumPrice);
			
			entity.setAreaCode(task.getAreaCode());
			entity.setAreaName(task.getAreaName());

			entity.setGrabChannelName(task.getGrabChannel());
			entity.setGrabChannelId(task.getGrabChannelId());
			if (!clzz.equals(ReturnDoublePlaneInfoEntity.class)) {
				entity.setFlightDate(task.getFlightDate());
				entity.setFromCity(task.getFromCity());
				entity.setFromCityName(task.getFromCityName());
				entity.setToCity(task.getToCity());
				entity.setToCityName(task.getToCityName());
			} else {
				entity.setFlightDate(task.getReturnGrabDate());
				entity.setFromCity(task.getToCity());
				entity.setFromCityName(task.getToCityName());
				entity.setToCity(task.getFromCity());
				entity.setToCityName(task.getFromCityName());
			}

			entity.setAttachHbaseKey(task.getAttachHbaseKey());
			if (entity instanceof DoublePlaneInfoEntity) {
				((DoublePlaneInfoEntity) entity).setFlightReturnDate(task.getReturnGrabDate());
			}
			
		return (T) entity;
	}
	
	/**
	 * 解析数据，返回航程信息
	 * @param trs
	 * @param clz
	 * @return
	 * @throws Exception
	 */
	private <T> List<T> htmlToTraintEntity(Elements trs,Class<T> clz) throws Exception{
		boolean isFrist=true;
		List<T> list=new ArrayList<T>();
		for(Element tr: trs){
			int i=0;
			Elements tds=tr.getElementsByTag("td");
			String flightDate=tds.get(i++).text();
			flightDate=flightDate.replace("年", "-").replace("月", "-").replace("日", "");
			
			String flightNo=tds.get(i++).text();	
			
			String carrierName =tds.get(i++).text();
			String carrierFullName=carrierName;
			
			String fromCityAndTime=tds.get(i++).text();
			String[] strs=fromCityAndTime.split(" ");
			String fromCity=strs[0];
			Date startTime=PlaneInfoEntityBuilder.getFlightTime(flightDate, strs[1]);
			
			String toCityAndTime=tds.get(i++).text();
			strs=toCityAndTime.split(" ");
			String toCity=strs[0];
			Date endTime=PlaneInfoEntityBuilder.getFlightTime(flightDate, strs[1]);
			if(strs.length>2)
				endTime=DateUtil.add(endTime, Calendar.DAY_OF_YEAR, 1);
			
			i++;
			
			Long milltimes=null;
			if(isFrist){
				String stayTime=tds.get(i++).text();
				
				if(null!=stayTime){
					String[] hm=stayTime.split(":");
					int hour =  Integer.valueOf(hm[0]);
					int minute =  Integer.valueOf(hm[1]);
					milltimes=(hour * 60 + minute) * 60 * 1000l;
				}
				isFrist=false;
			}
			
			String carrierKey=null;
			List<List<String>> carrierKeys = RegHtmlUtil.retrieveLinkss(tds.get(i).html(), this.getRegex());
			if(null!=carrierKeys && carrierKeys.size()>0){
				carrierKey=carrierKeys.get(0).get(1);
			}
			String flightType=tds.get(i++).text();
			
			String cabinName=tds.get(i++).text();
			if (TransitEntity.class.equals(clz)) {
				TransitEntity entity=PlaneInfoEntityBuilder.buildTransitEntity(flightNo, null, carrierKey, carrierName,
						carrierFullName, fromCity, null, toCity, null, flightType);
				entity.setStartTime(startTime);
				entity.setEndTime(endTime);
				entity.setStayTime(milltimes);
				entity.setCabinName(cabinName);
				list.add((T) entity);
			}else if (ReturnTransitEntity.class.equals(clz)){
				ReturnTransitEntity entity=PlaneInfoEntityBuilder.buildReturnTransitEntity(flightNo, null, carrierKey, carrierName,
						carrierFullName, fromCity, null, toCity, null, flightType);
				entity.setStartTime(startTime);
				entity.setEndTime(endTime);
				entity.setStayTime(milltimes);
				entity.setCabinName(cabinName);
				list.add((T) entity);
			}
		}
		return list;
	}
	

	/**
	 * 抓取步骤1。
	 * 
	 * @param cabinType
	 * @return
	 * @throws Exception
	 */
	private String fetchStep1(String cabinType) throws Exception {
		logger.info("抓取舱位"+cabinType+"的数据，步骤1");
		setContext();
		String url = "http://www.cathaypacific.com/wdsibe/IBEFacade.html";
		String page = null;
		Map<String, String> params = Maps.newHashMap();
		HttpPost post = null;
		HttpResponse response=null;
		HttpEntity entity =null;
		try {
			params.put("ENTRYPOINT", "http://www.cathaypacific.com:80/cx/sc_CN.html");
			params.put("ERRORURL", "http://www.cathaypacific.com:80/cx/sc_CN/book-a-trip/book-flights/book-flights-now/_jcr_content.handler.html");
			params.put("RETURNURL", "http://www.cathaypacific.com:80/cx/sc_CN/_jcr_content.handler.html");
			params.put("ACTION", "SINGLECITY_SEARCH");
			params.put("BOOKING_FLOW", "REVENUE");
			params.put("ENTRYCOUNTRY", "CN");
			params.put("ENTRYLANGUAGE", "sc");
			params.put("FLEXIBLEDATE", "false");
			params.put("ADULT", "1");
			params.put("CHILD", "0");
			params.put("INFANT", "0");
			params.put("DEPARTUREDATE", taskQueue.getFlightDate().replaceAll("-", ""));
			params.put("ARRIVALDATE", taskQueue.getReturnGrabDate().replaceAll("-", ""));
			params.put("ORIGIN", taskQueue.getFromCity());
			params.put("DESTINATION", taskQueue.getToCity());
			params.put("CABINCLASS", cabinType);
			params.put("TRIPTYPE", taskQueue.getIsReturn() == 1 ? "R" : "O");
			
			post=super.getBasePost(url, params);
			post.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			post.setHeader("Accept-Language", "zh-cn,en-us;q=0.8,zh;q=0.5,en;q=0.3");
		//	post.setHeader("Accept-Encoding","gzip, deflate");
			post.setHeader("Connection", "keep-alive");
			post.setHeader("DNT", "1");
			post.setHeader("Host", "www.cathaypacific.com");
			post.setHeader("Referer","http://www.cathaypacific.com/cx/sc_CN.html?brand=CX");
			post.setHeader("User-Agent", RandomBrowserVersion.getBV().getUserAgent());
			
			response=getHttpClient().execute(post,context);
			int statusCode=response.getStatusLine().getStatusCode();
			entity=response.getEntity();
			page = EntityUtils.toString(entity);
			super.appendPageContents(page);
			if(statusCode>=400)
				throw new PageErrorResultException(statusCode+"");
		} catch (Exception e) {
			Thread.sleep(1000*2);
			rollbackProxyIp(false);
			switchProxyipByHttClient();
			logger.info("抓取舱位[%s]的数据，步骤[1]失败：" + e.getMessage());
			throw e;
		} finally {
			if(null!=post)
				post.releaseConnection();
			if(null!=entity)
				EntityUtils.consumeQuietly(entity);
			params = null;
			response=null;
		}
		return fetchStep2(page); // 执行步骤2
	}

	/**
	 * 抓取步骤2。
	 * 
	 * @param cabinType
	 * @return
	 * @throws Exception
	 */
	private String fetchStep2(String page) throws Exception {
		logger.info("抓取舱位[%s]的数据，步骤2");
		Map<String, String> params = Maps.newHashMap();
		HttpPost post = null;
		HttpResponse response=null;
		HttpEntity entity =null;
		try {
			Document doc = Jsoup.parse(page);
			Element elem = doc.select("form[name=SubmissionDetails]").get(0);
			String action = elem.attr("action");
			Elements inputs = elem.getElementsByTag("input");
			for (Element e : inputs) {
				params.put(e.attr("name"), e.attr("value"));
			}
			post=getBasePost(action, params);
			response=getHttpClient().execute(post,context);
			int statusCode=response.getStatusLine().getStatusCode();
			entity=response.getEntity();
			page = EntityUtils.toString(entity);
			super.appendPageContents(page);
			if(statusCode>=400 && isUseProxyip()){
				post.releaseConnection();
				EntityUtils.consumeQuietly(entity);
				rollbackProxyIp(false);
				switchProxyipByHttClient();
				response=getHttpClient().execute(post);
				statusCode=response.getStatusLine().getStatusCode();
				entity=response.getEntity();
				page = EntityUtils.toString(entity);
				super.appendPageContents(page);
			}
			
			if(statusCode>=400)
				throw new PageErrorResultException(statusCode+"");
			if(null==page || "".equals(page))
				throw new FlightInfoNotFoundException("没有航班");
		} catch (Exception e) {
			Thread.sleep(1000*2);
			rollbackProxyIp(false);
			switchProxyipByHttClient();
			logger.info("抓取舱位[%s]的数据，步骤[2]失败：" + e.getMessage());
			throw e;
		} finally {
			params = null;
			if(null!=post)
				post.releaseConnection();
			if(null!=entity)
				EntityUtils.consumeQuietly(entity);
			response=null;
		}
		return page;
	
	}

	/**
	 * 抓取步骤3 _ 获取数据详细信息。
	 */
	private String fetchDetail(Element doc, String row1, String row2) throws Exception {
		String page = null;
		Map<String, String> params = Maps.newHashMap();
		HttpPost post = null;
		HttpResponse response=null;
		HttpEntity entity =null;
		try {
			String action = doc.attr("action");
			params = parseFormDataToMap(doc, "AVAI_FORM");
			params.put("ROW_1", row1);
			params.put("WDS_DATE_1", taskQueue.getFlightDate().replaceAll("-", "") + "0000");
			if(null!=row2){
				params.put("ROW_2", row2);
				params.put("WDS_DATE_2", taskQueue.getReturnGrabDate().replaceAll("-", "") + "0000");
			}
			
			post=getBasePost(action, params);
			response=getHttpClient().execute(post,context);
			entity=response.getEntity();
			page = EntityUtils.toString(entity);
			
			super.setLenghtCount(page.getBytes().length);
		//	super.appendPageContents(page); //太多不保存
		} catch (Exception e) {
			Thread.sleep(1000*2);
			rollbackProxyIp(false);
			switchProxyipByHttClient();
			logger.info("抓取舱位[%s][%s-%s]的详细数据失败：" + e.getMessage());
		} finally {
			params = null;
			if(null!=post)
				post.releaseConnection();
			if(null!=entity)
				EntityUtils.consumeQuietly(entity);
			response=null;
		}
		
		return page;
	}

	private Map<String, String> parseFormDataToMap(Element doc, String formName) {
		Map<String, String> params = Maps.newHashMap();
		Element elem = doc.select("form[name=" + formName + "]").get(0);
		Elements inputs = elem.getElementsByTag("input");
		for (Element e : inputs) {
			params.put(e.attr("name"), e.attr("value"));
		}
		return params;
	}

	@Override
	public boolean validateFetch(Object obj) throws Exception {
		if (obj == null) {
			throw new Exception("获取数据为空");
		}
		return true;
	}
	
	public String getRegex(){
		String regex = "<a.+?,'(.+?)'\\)";
		return regex;	
	}
	
	  private void setContext() {
		   context=HttpClientContext.create();
		    Registry<CookieSpecProvider> registry = RegistryBuilder
		        .<CookieSpecProvider> create()
		        .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
		        .register(CookieSpecs.BROWSER_COMPATIBILITY,
		            new BrowserCompatSpecFactory()).build();
		    context.setCookieSpecRegistry(registry);
		    context.setCookieStore( new BasicCookieStore());
		  }

	
}
